package engine

import (
    "log"
    "spider/concurrent2/fetcher"
    "spider/concurrent2/parser"
)

// worker
// 将获取内容与解析内容封装到worker方法里边
// 方便我们使用协程开多个worker
func worker(request parser.Request) (parser.ParseResult, error) {
    
    //  获取内容
    log.Printf("Fetching %s", request.Url)
    content, err := fetcher.Fetch(request.Url)
    if err != nil {
        log.Printf("Fetcher: error fetching url %s : %v", request.Url, err)
        return parser.ParseResult{}, nil
    }
    
    // 解析内容
    parseResult := request.Parser.Analysis(content)
    
    return parseResult, nil
}